# -*- coding: utf-8 -*-
'''
Created on 14-10-2012

@author: LONG HOANG GIANG
'''

import sys, os
sys.path.append(os.path.join(os.path.dirname(__file__), '../'))

from CrawlerLib2 import commonlib, html2text
from urlparse import urljoin
from Model.storymodelv2 import StoryModel
import re
import datetime
import workerpool


class AloBooks():
    
    _sUrl = ''
    _introChap = {'title': 'GIỚI THIỆU', 'content': ''}
    appendTitle = True
    
    def __init__(self, url):
        self._sUrl = url
        
    def get_chapters(self):
        tree = commonlib.loadweb(self._sUrl).build_tree()
        if tree == None: return
        data = []
        
        gtNode = tree.xpath("//div[@class='content node-forum']/div[1]")
        gtContent = html2text.html2text(commonlib.Etree.tostring(gtNode[0]), w100=True).strip()
        gtContent = re.sub(r"\n", "<br />", gtContent)
        gtContent = re.sub(ur"<strong><br /> *<br />", '', gtContent)
        gtContent = re.sub(ur"(( *<br[^>]*> *){3,})", "<br /><br />", gtContent)
        gtContent = "<b>{0}</b><br /><hr /><br />{1}".format(self._introChap['title'], gtContent)
        self._introChap['content'] = gtContent
        
        for item in tree.xpath("//div[contains(@id, 'book-navigation')]/ul/li/a"):
            href = commonlib.normalize_str(item.get('href', ''))
            if href == '': continue
            url = urljoin(self._sUrl, href)
            name = commonlib.stringify(item)
            data.append({'name': name, 'url': url})
            
        def load(item):
            commonlib.loadweb(item['url'])
            
        pool = workerpool.WorkerPool(size=5)
        pool.map(load, data)
        pool.shutdown()
        pool.wait()
            
        return data
    
    def detect_chapter_number(self, url):
        chapters = []
        if re.search(r"-chuong-\d+\.aspx", url):
            chapters = [commonlib.extractText(r"-chuong-(\d+)\.aspx", url, 1)]
        elif re.search(r"-chuong-\d+-\d+\.aspx", url):
            start = int(commonlib.extractText(r"-chuong-(\d+)-\d+\.aspx", url, 1))
            end = int(commonlib.extractText(r"-chuong-\d+-(\d+)\.aspx", url, 1))
            chapters = [i for i in range(start, end+1)]
        return chapters
    
    def get_content_chapter(self, item):
        url = item['url']
        tree = commonlib.loadweb(url).build_tree()
    
    def get_content2(self, item):
        url = item['url']
        tree = commonlib.loadweb(url).build_tree()
        contentNode = tree.xpath("//div[contains(@class, 'content node')]/div[1]")
        content = html2text.html2text(commonlib.Etree.tostring(contentNode), url).strip()
        content = re.sub(ur"\n", "<br />", content)
        data = []
        n = len(re.findall(r"<strong>(Chương[^<]+)</strong>".decode('utf-8'), content))
        idx1 = idx2 = 0
        for match in re.finditer(r"<strong>(Chương[^<]+)</strong>".decode('utf-8'), content):
            idx2 = match.start()
            if idx1 == idx2 == 0: continue
            icontent = content[idx1:idx2]
            title = commonlib.extractText(r"<strong>(Chương[^<]+)</strong>".decode('utf-8'), icontent, 1)
            data.append({'title': title, 'content': icontent})
            idx1 = idx2
        icontent = content[idx2:]
        title = commonlib.extractText(r"<strong>(Chương[^<]+)</strong>".decode('utf-8'), icontent, 1)
        data.append({'title': title, 'content': icontent})
        return data
    
    def get_content(self, item):
        url = item['url']
        tree = commonlib.loadweb(url).build_tree()
        data = []
        nodes = tree.xpath("//div[@class='content node-book']//h1[@class='entry-title']")
        if len(nodes) > 0:
            for item in nodes:
                title = commonlib.stringify(item)
                contentNode = item.xpath("./following-sibling::*[1]")
                content = html2text.html2text(commonlib.Etree.tostring(contentNode), url)
                title = re.sub(r"\(.+\)", "", title).strip()
                print title
                if self.appendTitle:
                    content = "<b>{0}</b><br /><br />{1}".format(title, content)
                data.append({'title': title, 'content': content})
            return data
        contentNode = tree.xpath("//div[contains(@class, 'content node')]/div[1]")
        content = html2text.html2text(commonlib.Etree.tostring(contentNode), baseurl=url, allow_image=False).strip()
        content = content.replace("previous page next page", '')
        title = commonlib.extractText(ur"(Chương \d+[^\n]+)", content, 1)
        title = re.sub(ur"<[^>]+>", '', title).strip()
        if '-' in item['name']:
            t = item['name']
            t = t[t.find('-')+1:]
            item['name'] = t.strip()
        title = title.replace('-', '').replace(':', '').strip()
        if title != '': item['name'] = title
        content = re.sub(ur"(Chương \d+[^\n]+)", '', content)
        content = re.sub(ur"( *\n *){3,}", "\n\n", content)
        if self.appendTitle:
            content = "<b>{0}</b><br /><hr /><br />{1}".format(item['name'], content)
        data.append({'title': item['name'], 'content': content})
        return data
    
    def process(self):
        
        chapters = self.get_chapters()
        storyModel = StoryModel(database_path)
        storyModel.open(True)
        storyModel.add_story(self._introChap['title'], self._introChap['content'])
        print '_________________________'
        print self._introChap['title']
        print '_________________________'
        print self._introChap['content']
        for item in chapters:
            
#            data = self.get_content2(item)
            data = self.get_content(item)
            for chapter in data:
                if chapter['title'].strip() == '': continue
                content = chapter['content'].replace(REPLACE, '').strip() if REPLACE != '' else chapter['content'].strip()
                content = re.sub(ur"\n", "<br />", content)
                content = re.sub(ur"<br /><br /><br />", "", content)
                content = re.sub(ur"<strong><br /> *<br />", '', content)
                print '--------------------------------'
                print chapter['title']
                print '--------------------------------'
                print content[:255]
                
                storyModel.add_story(commonlib.toUpper(chapter['title']), content)
        storyModel.close()
            
            

if __name__ == '__main__':
    
    database_path = '/longhoanggiang/database/{0}'.format('ayern')
    
    REPLACE = ""
    
    alobooks = AloBooks('http://alobooks.vn/forum/13722/anh-yeu-em-rat-nhieu-update-co-to-lan.aspx')
    alobooks.appendTitle = True
    alobooks.process()
    
    print '> Finished at {0}'.format(datetime.datetime.now())
    os._exit(1)
    